//xlog
package main

/*
  本程序是一个web访问日志分析工具, 以实用为目标, 属于自己开发使用的压箱底工具..
*/
import (
	"bufio"
	//"bytes"
	"flag"
	"fmt"
	"os"
	"xlog/color"
	"xlog/web"

	//"regexp"
	//"sort"
	//"strconv"
	"compress/gzip"
	"path"
	"runtime"
	"strings"
	"xlog/common"
)

var ipdatabase string

var GoVersion string
var BuildTime string

//var KernelVer string

// 主函数
func main() {
	// 处理命令行参数
	help := flag.Bool("help", false, "show help")
	help2 := flag.Bool("h", false, "show help")
	filter := flag.String("F", "", "set filter")
	column := flag.String("C", "", "set colums define for log data")
	summary := flag.Bool("s", false, "show or hide summary")
	data := flag.Bool("d", false, "show or hide log data")
	dformat := flag.String("df", "txt", "format:txt or json,use with -d")
	trend := flag.String("t", "", "show or hide trend")
	scode := flag.Bool("sc", false, "show detail StatusCode")
	uagent := flag.Bool("ua", false, "show detail UserAgent")
	requesturi := flag.String("uri", "", "show detail request uri")
	lineNo := flag.Bool("l", true, "show or hide line number for log data")
	logfmt := flag.String("f", "", "set log format, often no need")
	showColor := flag.Bool("c", true, "show or not show color")
	groupby := flag.String("g", "", "group by a field")
	vgroupbytotal := flag.Bool("vt", true, "groupbytotal")
	bytes := flag.Bool("B", false, "show or hide summary")
	top := flag.Int("top", 20, "show top result for option groupby")
	logfile := flag.String("log", "", "logfile path")
	port := flag.String("port", "9000", "listen port")
	//ipdatafile := flag.String("ipf", "/etc/xlog/ip2region.xdb", "ipdatabase path")
	flag.Parse()

	if *help || *help2 {
		Help()
		return
	}

	//trend 扩展参数
	var TrendExtendParameters string
	if *scode {
		TrendExtendParameters = "sc"
	} else if *uagent {
		TrendExtendParameters = "ua"
	} else if *requesturi != "" { //需要参数大于4个字节
		TrendExtendParameters = *requesturi
	}

	if flag.NFlag() == 0 {
		*summary = true
	}

	if *showColor {
		color.Enable()
	}

	// 行数统计变量
	//total := 0
	//filtered := 0

	var total int64 = 0
	var filtered int64 = 0

	// 合计总字节数
	var bodyBytesTotal int64 = 0
	var bodyBytesFiltered int64 = 0
	// 合计总响应时间
	var timeSecondsTotal float64 = 0.0
	var timeSecondsFiltered float64 = 0.0

	// 合计响应状态码分布
	var statusCodeCounterTotal = common.NewCounter()
	var statusCodeCounterFiltered = common.NewCounter()

	//统计字节groupby
	var bodyBytesCounterTotal = common.NewCounter()
	var bodyBytesCounterFiltered = common.NewCounter()

	// 合计Http方法分布
	var methodCounterTotal = common.NewCounter()
	var methodCounterFiltered = common.NewCounter()

	// 合计协议分布
	//var schemeCounterTotal = common.NewCounter()
	//var schemeCounterFiltered = common.NewCounter()

	// 合计响应时间分布
	var timeDistCounterTotal = common.NewCounter()
	var timeDistCounterFiltered = common.NewCounter()

	// 合计RealIP分布
	var realIPCounterTotal = common.NewCounter()
	var realIPCounterFiltered = common.NewCounter()

	// 合计Uri分布
	var uriCounterTotal = common.NewCounter()
	var uriCounterFiltered = common.NewCounter()

	// 合计Uri模式分布
	var uriPtCounterTotal = common.NewCounter()
	var uriPtCounterFiltered = common.NewCounter()

	// 合计UA分布
	var uaCounterTotal = common.NewCounter()
	var uaCounterFiltered = common.NewCounter()

	// 合计referer分布
	var rfCounterTotal = common.NewCounter()
	var rfCounterFiltered = common.NewCounter()

	var trendCounter = common.NewTrendCounter()

	var groupbyCounterTotal = common.NewCounter()
	var groupbyCounterFiltered = common.NewCounter()

	var log *common.Log

	var isGroupby = (*groupby != "")
	var groupFields = strings.Split(*groupby, ",")
	var groupbyKey string

	//demo 模式运行
	NCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(NCPU)
	if *port != "" && *logfile != "" {
		go web.BackupCounter(*logfile)
		web.Handle(*port)
	}

	//为|| 时为true
	var tjf bool
	// 初始化过滤条件并显示提示
	var filters []*common.Filter
	if *filter == "" {
		if *summary {
			fmt.Println("No filter")
		}
	} else {
		if *summary {
			fmt.Println("Filter: ", *filter)
		}
		filters = make([]*common.Filter, 0)
		//fx := strings.Split(*filter, " && ")
		var fx []string
		if *filter != "" {
			if strings.Contains(*filter, "||") {
				tjf = true
				fx = strings.Split(*filter, " || ")
			} else {
				fx = strings.Split(*filter, " && ")
			}
		}
		for _, fy := range fx {
			f := common.NewFilter(fy)
			filters = append(filters, f)
		}
	}

	// 打开文件并处理失败
	var file *os.File
	var bi *bufio.Reader
	if flag.Arg(0) != "" {
		var err interface{}
		file, err = os.Open(flag.Arg(0))
		if err != nil {
			fmt.Println(err)
			return
		}
		defer file.Close()
		if path.Ext(flag.Arg(0)) == ".gz" {
			gfile, err := gzip.NewReader(file)
			if err != nil {
				fmt.Println(err)
				return
			}
			defer gfile.Close()
			bi = bufio.NewReader(gfile)
		} else {
			bi = bufio.NewReader(file)
		}
	} else {
		file = os.Stdin
		defer file.Close()
		bi = bufio.NewReader(file)
	}
	//defer file.Close()

	// 建立行式文件缓冲流
	//bi := bufio.NewReader(file)

	parser := common.NewLogParser(*logfmt)
	// 遍历日志文件进行统计
	for {
		line, err := bi.ReadString('\n')
		if err != nil {
			break // 读取完成整个文件退出循环
		}

		// 使用合适的解析器解析日志
		log = parser.ParseLog(line)
		if log == nil {
			continue
		}

		total++

		var timeDist string
		if *summary {
			// 累加总字节数和响应时间
			bodyBytesTotal = bodyBytesTotal + log.BodyBytes
			timeSecondsTotal = timeSecondsTotal + log.TimeSeconds

			// 处理总响应状态码分布
			statusCodeCounterTotal.Count(fmt.Sprint(log.StatusCode))

			// 处理总Http方法分布
			methodCounterTotal.Count(log.Method)

			// 处理总协议分布
			//schemeCounterTotal.Count(log.Scheme)

			// 处理总响应时间分布
			timeDist = log.TimeDist("t")
			timeDistCounterTotal.Count(timeDist)

			// 处理总RealIP分布
			realIPCounterTotal.Count(log.XRealIP)

			// 处理总Uri分布
			uriCounterTotal.Count(log.Uri)

			// 处理总Uri模式分布
			uriPtCounterTotal.Count(log.UriPattern)

			// 处理总UA分布
			uaCounterTotal.Count(log.UserAgent)

			// 处理总Referer分布
			rfCounterTotal.Count(log.Referer)
		}

		//var groupbyKey string
		if isGroupby {
			groupbyKey = log.Groupby(groupFields)
			groupbyCounterTotal.Count(groupbyKey)
			if *bytes {
				//for groupby 统计总的字节
				bodyBytesTotal = bodyBytesTotal + log.BodyBytes
				bodyBytesCounterTotal.CountBytes(log, groupbyKey)
			}
		}

		// 处理过滤条件
		if len(filters) > 0 {
			if common.MatchFilters(filters, log, tjf) {
				filtered++

				if *summary {
					// 累加过滤字节数和响应时间
					bodyBytesFiltered = bodyBytesFiltered + log.BodyBytes
					timeSecondsFiltered = timeSecondsFiltered + log.TimeSeconds

					// 处理过滤的响应状态码分布
					statusCodeCounterFiltered.Count(fmt.Sprint(log.StatusCode))

					// 处理过滤的Http方法分布
					methodCounterFiltered.Count(log.Method)

					// 处理过滤的协议分布
					//schemeCounterFiltered.Count(log.Scheme)

					// 处理过滤的响应时间分布
					timeDistCounterFiltered.Count(timeDist)

					// 处理过滤的XRealIP分布
					realIPCounterFiltered.Count(log.XRealIP)

					// 处理过滤的Uri分布
					uriCounterFiltered.Count(log.Uri)

					// 处理过滤的Uri模式分布
					uriPtCounterFiltered.Count(log.UriPattern)

					//处理过滤的UA分布
					uaCounterFiltered.Count(log.UserAgent)

					//处理过滤的Referer分布
					rfCounterFiltered.Count(log.Referer)
				}

				if isGroupby {
					groupbyKey = log.Groupby(groupFields)
					groupbyCounterFiltered.Count(groupbyKey)
					if *bytes {
						//统计过滤条件的字节
						bodyBytesFiltered = bodyBytesFiltered + log.BodyBytes
						bodyBytesCounterFiltered.CountBytes(log, groupbyKey)
					}
				}

				if *data {
					if *dformat == "json" {
						log.DisplayColumnJson(*column, line, filtered, *lineNo)
					} else {
						log.DisplayColumn(*column, line, filtered, *lineNo)
					}
				}

				if *trend != "" {
					trendCounter.Count(log, *trend, TrendExtendParameters)
				} else {
					trendCounter.Count(log, "m", TrendExtendParameters)
				}
			}
		} else {
			if *data {
				if *dformat == "json" {
					log.DisplayColumnJson(*column, line, total, *lineNo)
				} else {
					log.DisplayColumn(*column, line, total, *lineNo)
				}
			}
			if *trend != "" {
				trendCounter.Count(log, *trend, TrendExtendParameters)
			} else {
				trendCounter.Count(log, "m", TrendExtendParameters)
			}
		}

	}

	if *summary {
		// 打印日志行数合计
		color.Cyan("Log  count   total: ")
		color.Yellow("%20d, ", total)
		color.Cyan("filtered: ")
		color.Yellow("%20d, ", filtered)
		color.Cyan("rate: ")
		color.Green("%.2f%%\n", float64(filtered)/float64(total)*100)

		// 打印日志字节数合计
		color.Cyan("Body bytes   total: ")
		color.Yellow("%17d MB, ", bodyBytesTotal/1024/1024)
		color.Cyan("filtered: ")
		color.Yellow("%17d MB, ", bodyBytesFiltered/1024/1024)
		color.Cyan("rate: ")
		color.Green("%0.2f%%\n", float64(bodyBytesFiltered)/float64(bodyBytesTotal)*100)

		// 打印日志响应时间合计
		color.Cyan("Time seconds total: ")
		color.Yellow("%19.2fs, ", timeSecondsTotal)
		color.Cyan("filtered: ")
		color.Yellow("%19.2fs, ", timeSecondsFiltered)
		color.Cyan("rate: ")
		color.Green("%0.2f%%\n", timeSecondsFiltered/timeSecondsTotal*100)

		// 打印平均响应时间
		color.Cyan("Time average total: ")
		color.Yellow("%19.2fs, ", timeSecondsTotal/float64(total))
		color.Cyan("filtered: ")
		color.Yellow("%19.2fs\n", timeSecondsFiltered/float64(filtered))

		// 打印响应时间分布
		color.Cyan("Time dist    total:\t\t\t  filtered:\n")
		common.PrintMap2(timeDistCounterTotal.Map, total, timeDistCounterFiltered.Map, filtered)

		// 打印状态码分布
		color.Cyan("Status code  total:\t\t\t  filtered:\n")
		common.PrintMap2(statusCodeCounterTotal.Map, total, statusCodeCounterFiltered.Map, filtered)

		// 打印Http方法分布
		color.Cyan("Http method  total:\t\t\t  filtered:\n")
		common.PrintMap2(methodCounterTotal.Map, total, methodCounterFiltered.Map, filtered)

		// 打印协议分布
		//color.Cyan("request scheme total:\t\t\t  filtered:\n")
		//PrintMap2(schemeCounterTotal.Map, total, schemeCounterFiltered.Map, filtered)

		// 打印RealIP分布
		color.Cyan("RealIP       total:\t\t\t  filtered:\n")
		common.PrintMap2(realIPCounterTotal.Map, total, realIPCounterFiltered.Map, filtered)

		// 打印Uri分布
		color.Cyan("Uri total:\n")
		common.PrintMap(uriCounterTotal.Map, total, 10, false)
		color.Cyan("Uri filtered:\n")
		common.PrintMap(uriCounterFiltered.Map, filtered, 10, false)

		// 打印Uri模式分布
		//color.Cyan("Uri pattern total:\n")
		//PrintMap(uriPtCounterTotal.Map, total, 20, false)
		//color.Cyan("Uri pattern filtered:\n")
		//PrintMap(uriPtCounterFiltered.Map, filtered, 20, false)

		// 打印UA分布
		color.Cyan("UA total:\n")
		common.PrintMap(uaCounterTotal.Map, total, 10, false)
		color.Cyan("UA filtered:\n")
		common.PrintMap(uaCounterFiltered.Map, filtered, 10, false)

		// 打印Referer分布
		color.Cyan("Referer total:\n")
		common.PrintMap(rfCounterTotal.Map, total, 10, false)
		color.Cyan("Referer filtered:\n")
		common.PrintMap(rfCounterFiltered.Map, filtered, 10, false)
	}

	if isGroupby {
		if *bytes {
			color.Cyan("bodybytes filtered:\n")
			common.PrintMap(bodyBytesCounterFiltered.Map, bodyBytesFiltered, *top, true)
			color.Cyan("Groupby total:\n")
			common.PrintMap(bodyBytesCounterTotal.Map, bodyBytesTotal, *top, true)
		} else {
			color.Cyan("Groupby pattern filtered:\n")
			common.PrintMap(groupbyCounterFiltered.Map, filtered, *top, false)
			if *vgroupbytotal {
				color.Cyan("Groupby total:\n")
				common.PrintMap(groupbyCounterTotal.Map, total, *top, false)
			}
		}
	}

	if *trend == "m" || *trend == "s" || *trend == "d" || *trend == "hour" {
		if *scode {
			color.Cyan("Time\tAll\tIp\tAvgSeconds\tMaxseconds\tBodyBytes[MB]\t200\t301\t302\t400\t403\t404\t499\t500\t502\t503\t504\n")
		} else if *uagent {
			//color.Cyan("Time\tAll\tIp\tAvgSeconds\tMaxseconds\tBodyBytes[MB]\tIE\tchrome\tfirefox\tsafari\tBaiduspider\t360Spider\tSogouspider\tGooglebot\tSosospider\tYisouSpider\n")
			//color.Cyan("Time\tAll\tIE\tchrome\tfirefox\tsafari\tBaidu\t360\tSogou\tGoogle\tSoso\tYisou\n")
			color.Cyan("Time\tAll\tBaidu\t360\tSogou\tGoogle\tSoso\tYisou\t")
			color.Cyan("UC\tWeiXin\tMQQ\t")
			color.Cyan("Maxthon\tQQ\tLieBao\tSE360\t")
			color.Green("msie\tchrome\tfirefox\tsafari\tOtherUa\n")
		} else if len(*requesturi) > 4 {
			uarry := strings.Split(*requesturi, ",")
			color.Cyan("Time\tAll\t")
			for k := 1; k <= len(uarry); k++ {
				color.Cyan("URI%d\t", k)
			}
			color.Cyan("\n")
		} else {
			color.Cyan("Time\tAll\tIp\tAvgSeconds\tMaxseconds\tBodyBytes[MB]\t200\t30x\t40x\t50x\n")
		}
		trendCounter.Print(TrendExtendParameters)
	} else {
		if *trend != "" {
			aa := "key"
			color.Cyan("%80s\tAll\tIp\tAvgSeconds\tBodyBytes[MB]\t200\t30x\t40x\t50x\n", aa)
			trendCounter.Print2()
		}
	}
}

func Help() {
	color.Enable()
	fmt.Printf("\nVersion: v3.6")
	fmt.Printf("\nBuildTime: " + BuildTime)
	fmt.Printf("\nGoLangVer: " + GoVersion)
	//fmt.Printf("\nKernelVer: " + KernelVer)
	fmt.Println("\nUsage: cat access.log | ./xlog -F=\"s = 200 && m = GET && r rp ^/f[0-9].html\" -t=m")
	color.Cyan("\nFilter fields:\n")
	fmt.Println("\tBackHost        (U) nginx log only, upstream, maybe more than one")
	fmt.Println("\tBackStatus      (S) nginx log only, one status per upstream, maybe more than one")
	fmt.Println("\tBackTimeSeconds (z) nginx log only, one time per upstream, maybe more than one")
	fmt.Println("\tBodyBytes       (b) response body bytes, maybe 0")
	fmt.Println("\tHost            (H) http header \"Host\", most time is domain, eg: bbs.pcauto.com.cn")
	fmt.Println("\tLocalAddr       (l) nginx log only, IP address of nginx server self")
	fmt.Println("\tMethod          (m) http method of request: GET, POST, HEAD ...")
	fmt.Println("\tMinute          (M) log time in minute, eg. 12:03, 01:45")
	fmt.Println("\tReferer         (e) http header \"Referer\"")
	fmt.Println("\tRemoteAddr      (h) ip address direct connect to nginx server, maybe other proxy server")
	fmt.Println("\tCremoteAddr     (n) ip address C net;eg:192.168.1")
	fmt.Println("\tSession         (c) cookie \"common_session_id\" of passport")
	fmt.Println("\tStatusCode      (s) status code, eg. 200, 301, 400, 404, 500, 502, 503, 504 ...")
	fmt.Println("\tTimeLocal       (T) request time at, eg. 05/Mar/2014:20:57:01")
	fmt.Println("\tTimeSeconds     (t) seconds from the request receive to response finished")
	fmt.Println("\tUri             (r) request uri with query string, eg. /forum.do?fid=250&pageNo=2")
	fmt.Println("\tUriPattern      (p) replace the number of uri to [0-9], eg. /forum.do?fid=[0-9]&pageNo=[0-9]")
	fmt.Println("\tRawUri          (i) Cancel ? in Uri \"RawUri\"")
	fmt.Println("\tUserAgent       (a) http header \"User-Agent\"")
	fmt.Println("\tVersion         (v) http version, eg, HTTP/1.0, HTTP/1.1")
	fmt.Println("\tXForwardFor     (x) http header \"X-Forwarded-For\"")
	fmt.Println("\tXRealIP         (R) http header \"X-Real-IP\"")
	fmt.Println("\tXRealcIP        (CR) http header c\"X-Real-IP\"")
	fmt.Println("\tHitStatus       (k) cache server HIT status")
	fmt.Println("\tScheme          (w) http or https")
	fmt.Println("\tProvince        (pro) Province")
	fmt.Println("\tCity            (city) City")
	fmt.Println("\tNetOperator     (net) NetOperator")

	color.Cyan("\nFilter operators: (-F)\n")
	color.Cyan("-F")
	fmt.Println(" is the option most often to use. use \"field1 op value1 && ... && fieldN op valueN\" to filter what you want.")

	fmt.Println("\t=  !=  >  >=  <  <=  (for all fields)")
	fmt.Println("\tstartWith(sw)     (for string fields only,eg:-F=\"r sw /index\")")
	fmt.Println("\tendWith(ew)       (for string fields only,eg:-F=\"r ew .html\")")
	fmt.Println("\tcontains(cs)      (for string fields only,eg:-F=\"r cs index\")")
	fmt.Println("\tnotcontains(ncs)  (for string fields only,eg:-F=\"r ncs index\")")
	fmt.Println("\tregexp(rp)        (for string or statuscode,eg:-F=\"r rp /v[2,3]/(cms|bbs)\")")
	fmt.Println("\tand(&&)           (for string or statuscode,eg:-F=\"a cs baidu && e cs baidu)")
	fmt.Println("\tor(||)            (for string or statuscode,eg:-F=\"a cs baidu || e cs baidu)")

	color.Cyan("\nOther options:\n")
	fmt.Println("\t(-t)    show group data per minute,eg: -t=m or -t=s or -t=hour -t=d,-t=a")
	fmt.Println("\t(-sc)   show detail StatusCode,must use with -t eg: -t=m -sc")
	fmt.Println("\t(-ua)   show detail Useragent,must use with -t eg: -t=m -ua")
	fmt.Println("\t(-uri)   show detail Useragent,must use with -t eg: -t=m -uri=\"/a.html,/b/,/cc/\"")
	fmt.Println("\t(-g)    groupby one or more fields, eg: -g=s or -g=a,s")
	fmt.Println("\t(-top)  show top result of groupby option,eg:-top=40")
	fmt.Println("\t(-s)    show report summary data,eg:-s=false,default is true")
	fmt.Println("\t(-c)    linux only, set false to save report in text file,-c=false")
	fmt.Println("\t(-d)    use for filter log only eg: -d")
	fmt.Println("\t(-df)   use with -d,show log format eg: -df=json or -df=txt")
	fmt.Println("\t(-vt)   show groupby total,eg:-vt=0")
	fmt.Println("\t(-C)    must use with -d to select fields in result,eg: -d -C=s or -d -C=h,s")
	fmt.Println("\t(-l)    must use with -d to use or not use line number,eg: -d -l=false")
	fmt.Println("\t(-B)    must use with -g to select fields count bodybytes,eg: -B -g=H")
	fmt.Println("\t(-f)    user define logfmt,logfmt defing in xlog.cfg,eg: -f=ats")
	fmt.Println("\t(-log)  run xlog realtime analyzer in demo,eg: -log=\"access.log\"")
	fmt.Println("\t(-port) define port,used with -log args,eg:-port=9000")
	//fmt.Println("\t(-ngxposter)  count post request[backup host 172.23.13.254:80 not count],default is false ,eg:-ngxposter=true")

	color.Cyan("\nInstallation instructions:\n")
	fmt.Println("\tconfig:  Please copy xlog.cfg to /etc/xlog/")
	fmt.Println("\tIP Library:  Please copy ipdata/ip2region.xdb to /etc/xlog/;ipLibrary Update:https://gitee.com/lionsoul/ip2region")
	//fmt.Println("\txlog.cfg eg: ats=(?P<XRealIP>\\S+) \\| \\[(?P<TimeLocal>\\S+) \\+0800] \\|)")
}
